{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# 精准率和召回率的平衡"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import datasets\n",
    "digits = datasets.load_digits()\n",
    "X = digits.data\n",
    "y = digits.target.copy()\n",
    "\n",
    "# 把数据变为极度偏斜的数据\n",
    "# 把手写数字分为9和非9两大类， 重点关注的是分类为9的数字\n",
    "y[digits.target==9] = 1\n",
    "y[digits.target!=9] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection._split import train_test_split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=666)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model.logistic import LogisticRegression\n",
    "\n",
    "log_reg = LogisticRegression()\n",
    "log_reg.fit(X_train, y_train)\n",
    "y_predict = log_reg.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.86746987951807231"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics.scorer import f1_score\n",
    "\n",
    "f1_score(y_test, y_predict)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "对于极其偏斜的数据，f1_score是比准确率更加准确的评判标准"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[403,   2],\n       [  9,  36]])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics.classification import confusion_matrix\n",
    "\n",
    "confusion_matrix(y_test, y_predict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-22.05698982, -33.02941569, -16.21335556, -80.37917862,\n       -48.25125007, -24.54006981, -44.39168237, -25.04296365,\n        -0.97828044, -19.71746541, -66.25139253, -51.09604623,\n       -31.49349131, -46.05333357, -38.67878372, -29.804725  ,\n       -37.58850985, -82.57570471, -37.81905611, -11.01165092,\n        -9.17440401, -85.13005039, -16.71614385, -46.23727242,\n        -5.32995324, -47.91762196, -11.66731351, -39.19611282,\n       -25.25295477, -14.36646658, -16.99785232, -28.91907577,\n       -34.33943416, -29.4761261 ,  -7.85813152,  -3.82091535,\n       -24.08168877, -22.16362583, -33.61223199, -23.14024822,\n       -26.91806299, -62.38939364, -38.85693085, -66.77261823,\n       -20.14483308, -17.47886804, -18.0680003 , -22.22227201,\n       -29.62306026, -19.7317084 ,   1.49551879,   8.32082289,\n       -36.29315563, -42.50734136, -25.90458854, -34.9896239 ,\n        -8.42013899, -50.04728023, -51.48209551,  19.88961877,\n        -8.91888431, -31.9934551 , -11.66101689,  -0.47143604,\n       -49.16131935, -46.23810032, -25.05390572, -19.61349165,\n       -36.16656665,  -3.12535392,  -3.91417768, -19.06042644,\n       -21.03315579, -41.52250775, -12.00622313, -33.89279525,\n       -35.84805068, -30.60482069, -56.51646499, -18.4547144 ,\n         4.51540405, -17.21608712, -76.6509628 , -58.54522882,\n       -31.72097005, -29.90832352, -33.31898806,  -9.08750913,\n       -47.64448242, -66.15303401, -16.95630795, -22.24904496,\n       -11.48960495, -18.10556765, -68.65401539, -47.02579261,\n       -40.11867546, -35.50213093, -17.19766207, -63.1028309 ,\n       -16.95448032, -55.10240983, -28.71258604, -68.81581771,\n       -68.31021284,  -6.25934259, -25.84001682, -38.00879959,\n       -27.90916803, -15.44712489, -27.45896655, -19.59774028,\n        12.33461216, -23.03868105, -35.94462968, -30.02831661,\n       -70.06675907, -29.48737389, -52.98827049, -24.9701781 ,\n       -12.32844683, -48.00990963,  -2.49966832, -59.92451798,\n       -31.1811674 ,  -8.65729993, -71.34895979, -57.01120864,\n       -21.09869424, -21.53851239, -69.34310449, -18.63522213,\n       -39.91440128, -57.26579871,  -0.84508123, -21.88381365,\n       -22.64112122, -29.21266229, -35.15703239, -20.25856769,\n       -11.40286777,   3.87278418,   6.0902311 ,   1.42891491,\n        -7.82713176, -39.3517755 ,  12.21055273, -75.10175581,\n       -75.38154073, -50.41809147, -11.55438797, -48.45868709,\n       -75.44079525, -29.98056353, -64.115748  ,  -7.16583808,\n        -6.52454459, -18.97252783, -33.71622096, -17.76221336,\n       -45.59384768, -33.53735769, -34.0869176 , -73.31507712,\n       -15.43459297,  12.16751627, -56.45927023,  -6.03194145,\n       -49.08445226, -16.54211799,  -2.05948386, -11.81042984,\n       -33.47406418, -50.77177798, -10.62905844, -17.67503306,\n        -5.07829806, -25.25781022, -16.61518598,   3.91126391,\n       -46.75608335, -12.89879513, -25.74790335, -16.31798628,\n       -23.55106273, -83.48233795,  -6.23507369, -19.83967242,\n       -20.06237028, -26.65466698, -27.11278842, -39.63713987,\n       -39.81307735, -27.43665472, -24.11826163, -21.24522141,\n       -10.49820521, -19.39896912, -41.95762702, -43.62362026,\n       -16.06839848, -64.09610465, -24.75461039, -56.57387368,\n       -13.50009133, -30.01580171,   3.93720227, -44.71703912,\n        -8.69366977,   1.58880115,  -2.76249701, -11.91888764,\n         7.58789186,  -7.25889104, -46.73813621, -49.19661382,\n        -4.80425843, -19.61031578, -24.30541692, -48.98794688,\n       -14.98137747, -24.83600505, -16.93957712, -19.46789085,\n       -15.77209189, -17.00120566, -39.23699498, -31.3745848 ,\n        -9.42200756, -71.38162563, -22.17497956, -14.72988062,\n       -23.57986286, -34.49384991,  -1.17647696, -32.90824422,\n       -10.82270715, -18.26228626,  -8.29315365, -44.84198538,\n       -22.59252369, -61.73626556, -47.12981702, -65.62590396,\n       -33.36447586, -24.00479739, -29.3316721 , -65.22706302,\n         1.43988098,  -4.56088187, -25.25851874, -22.46486506,\n       -54.43076487, -16.81739787, -11.2876251 , -35.25842526,\n        -5.57318783, -14.93094746, -70.95368223,  -6.50501407,\n        -1.22948688, -37.8755121 , -23.68948014, -68.29966191,\n        14.9380112 , -62.55689179,  10.14792903, -24.44798505,\n       -32.8538092 , -14.32958174, -85.68609977, -13.16400905,\n         9.27789929, -17.32728077, -36.06509683, -17.04719638,\n       -19.71314886, -32.72645554,  -5.36345665,   7.68319264,\n         9.20404568,   5.7653455 , -35.9635508 , -13.02391215,\n       -54.87489213, -41.61766813,   5.93733659, -79.11923251,\n       -16.01402104, -19.72190761, -10.96330708, -42.55203208,\n       -19.70964297, -16.20505932, -18.6873437 , -17.94403266,\n        -7.17465295, -20.54728583, -16.8107237 , -70.6903076 ,\n        -9.81780187, -32.87047298, -18.97777002, -21.3792665 ,\n       -25.15053177, -17.10990142, -13.5237327 , -23.76122525,\n        11.36505577, -14.50017756, -33.863127  , -13.71702625,\n       -50.52176002, -20.26636121, -56.12702414, -29.24278271,\n       -22.10083334, -31.39323367, -68.99342633, -60.34420664,\n        14.35288668,   8.69505982, -25.31395887,   2.38293893,\n         5.04571329, -19.56494466, -59.19927115, -10.05792063,\n       -29.66217456, -27.40199243,   6.13013253, -80.46966856,\n       -34.87544695, -49.84649642, -36.03967351, -48.50248758,\n       -19.9681346 , -62.0577459 ,  -3.237963  , -25.32912952,\n       -65.14035013,  -9.4273478 , -23.31751768,  19.38630237,\n       -18.84546992,  -4.47309603, -13.77212667, -21.88095993,\n       -43.4139309 , -51.85060636, -28.8391751 , -13.90473114,\n        -2.51950273,  -6.16015499,   3.14866284, -15.33995385,\n       -41.16632313, -25.89749645,  14.3019721 , -17.88821485,\n        14.6746669 , -33.65787901,   4.82445403, -14.42661096,\n       -54.22949572, -50.49133554, -30.54688302, -38.72569274,\n       -23.46182163, -24.87723649, -14.50557038, -23.72464018,\n       -28.07010585, -19.63716795, -28.66188364, -20.37698342,\n       -32.16755971, -11.15575571, -17.95929816, -24.54358664,\n       -24.60830566,  10.73693112, -16.68575851, -38.50781005,\n       -15.87671358, -37.05252066, -15.79373692, -68.69483064,\n       -33.64816977, -43.60839192, -28.74759712,  -9.88981785,\n       -67.16456328, -33.49890956, -45.89917029, -14.36734224,\n       -38.29001906, -14.76247524, -70.44234501, -11.19632535,\n       -41.46524889, -32.38986748, -20.86080131, -27.68977808,\n       -16.0608104 , -31.96317786,  -8.48422105, -22.10453367,\n       -34.0602908 , -12.47057702, -36.15119859, -36.5796647 ,\n       -22.46158873,   4.47537823, -20.8076859 ,  -3.75030678,\n       -20.31645961, -32.67831187, -41.10710864, -25.4602258 ,\n       -19.73669504, -47.83299293, -29.8578608 , -45.24588733,\n       -71.65704616,  -5.93559853, -32.93702275,   1.89661845,\n        11.76387911,   7.35783039, -30.93186667, -63.94241898,\n       -23.41431283,  -5.43423827, -33.46413418, -24.11265729,\n       -67.49717091, -34.30063954, -34.23323311, -31.61590145,\n       -52.86795879, -22.89223157,  -8.16020421, -17.73978454,\n       -26.9868432 , -32.38772585, -28.96088975, -67.25182327,\n       -46.4955085 , -16.11287066])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "log_reg.decision_function(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-22.05698982, -33.02941569, -16.21335556, -80.37917862,\n       -48.25125007, -24.54006981, -44.39168237, -25.04296365,\n        -0.97828044, -19.71746541])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 前10个都是负数\n",
    "log_reg.decision_function(X_test)[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 所以前10个的分类都为0\n",
    "log_reg.predict(X_test)[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 使用skLearn调整精准率-召回率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "decision_scores = log_reg.decision_function(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(-85.686099770956332, 19.889618774310772)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.min(decision_scores), np.max(decision_scores)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 调整threshold的值，默认是0，现在增大到5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_predict_2 = np.array(decision_scores >= 5, dtype=int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[404,   1],\n       [ 21,  24]])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_test, y_predict_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics.scorer import precision_score, recall_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.94736842105263153, 0.80000000000000004)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 调整前的精准率和召回率\n",
    "precision_score(y_test, y_predict), recall_score(y_test, y_predict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.95999999999999996, 0.53333333333333333)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 调整后的精准率和召回率\n",
    "precision_score(y_test, y_predict_2), recall_score(y_test, y_predict_2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "可以看到threshold增大后，精准率增加了，召回率减少了"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 把threshold的值由默认的0减少为-5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_predict_3 = np.array(decision_scores >= -5, dtype=int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.94736842105263153, 0.80000000000000004)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 调整前的精准率和召回率\n",
    "precision_score(y_test, y_predict), recall_score(y_test, y_predict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.72727272727272729, 0.88888888888888884)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 调整后的精准率和召回率\n",
    "precision_score(y_test, y_predict_3), recall_score(y_test, y_predict_3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "可以看到threshold减少后，精准率下降，召回率提高了"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
